/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */

package bbd.crawler.media;

import bbd.crawler.media.fwk.BaseURLMediaInterface;
import bbd.crawler.util.Console;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLDecoder;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArraySet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 *
 * @author Harit
 */
public class BaseMediaHandler implements BaseURLMediaInterface {

    private static final String HTTP_URL_REGEX = "<a href=\"[\\w \\d \\s [ / . & _ \\- ~ & %]]+\"";
    private static final String REMOVE_HREF_TAG = "<a href=\"";
    private static final String REMOVE_END_QUOTE = "\"$";
    private static final String REPLACE_WITH_NOTHING = "";
    private static final String SPLIT_WITH_SPACE = " ";
    private static final String HTTP_START_TAG = "http://";
    private static final String RELATIVE_URL_START = "/";

    private static final List<String> AUDIO_CONTENT_TYPE = Arrays.asList("audio/mpeg","audio.ogg","audio/prs.sid",
                                                                            "audio/x-ms-wma","audio/basic","audio-x-wav");
    private static final List<String> VIDEO_CONTENT_TYPE = Arrays.asList("video/x-ms-wmv","video/x-msvideo",
                                                                            "video/mpeg","video/quicktime","video/dv");
    private static final List<String> HTML_CONTENT_TYPE = Arrays.asList("text/html;charset=ISO-8859-1","text/html;charset=UTF-8",
                                                                            "text/html; charset=UTF-8","text/html","text/html; charset=iso-8859-1",
                                                                            "text/html; charset=iso-8859-1","text/html; charset=utf-8","text/plain; charset=UTF-8");
    
    private static final int TIMEOUT_MILLISECONDS = 3000;
    private static final int SLEEP_MILLISECONDS = 2000;
    private static String searchString = null;

    static Set<String> URLsSearched = new HashSet<String>();
    static Set<String> audioUrls = new HashSet<String>();
    static Set<String> videoUrls = new HashSet<String>();

    //--------------------------------------------------------------------------
    /** Javadoc here */
    public BaseMediaHandler () {} // end default constructor

    //--------------------------------------------------------------------------
    /** Javadoc here */
    public void query ( String query ) {

        throw new UnsupportedOperationException ("Not yet implemented");

    } // end function query

    //--------------------------------------------------------------------------
    /** Javadoc here */
    public Object getData () {

        throw new UnsupportedOperationException ("Not yet implemented");

    } // end function getData

    //--------------------------------------------------------------------------
    /** Javadoc here */
    protected static void getUrls ( CopyOnWriteArraySet<String> URLsToSearch, String searchString ) {
        
        String readLine = null;

        Pattern httpURLPattern = Pattern.compile( HTTP_URL_REGEX );

        Iterator iter = URLsToSearch.iterator();

        System.out.println("Size of URLstoSearch : " + URLsToSearch.size() );

        while ( iter.hasNext() ) {

            String baseUrlString = ( String ) iter.next();

            /**
             *  adding to list of searched
             *  Set and removing from current
             *  Set
             */
            URLsSearched.add( baseUrlString );
            URLsToSearch.remove( baseUrlString );

            //Console.message("Processing : " + baseUrlString );
            //Console.message("--------------------------------------------------");

            try {

                URL url = new URL ( baseUrlString );

                URLConnection connection = url.openConnection();

                connection.setReadTimeout( TIMEOUT_MILLISECONDS );

                BufferedReader reader = new BufferedReader ( new InputStreamReader ( connection.getInputStream() ) );

                while ( ( readLine = reader.readLine() ) != null ) {

                    Matcher httpURLMatcher = httpURLPattern.matcher( readLine );

                    if ( httpURLMatcher.find() ) {

                        String urlFound = httpURLMatcher.group().toString().
                                            replaceAll( REMOVE_HREF_TAG , REPLACE_WITH_NOTHING ).
                                            replaceAll( REMOVE_END_QUOTE , REPLACE_WITH_NOTHING );


                        urlFound = getAbsoluteURLName ( baseUrlString, urlFound );

                        /* validating if url contains search String */
                        if ( !doesURLContainsSearchString ( urlFound ) ) continue;

                        boolean recrawlURL = getURLContentType ( urlFound ) ;

                        if ( recrawlURL ) {

                            if ( ! ( URLsToSearch.contains( urlFound ) || URLsSearched.contains( urlFound ) ) ) {

                                    /**
                                     *  With this we are making sure
                                     *  that this is new URL to crawl
                                     */
                                    URLsToSearch.add( urlFound );
                                    System.out.println("added : " + urlFound );

                            } else {

                                //System.out.println("url ignore : " + urlFound );
                            }

                        }
                       
                    }
                    
                }

            } catch ( Exception e ) {

                Console.error( "BaseMediaHandler.getUrls", "JavaException - " + e.getMessage() );

            }

        }

        System.out.println(":Size of URLstoSearch NOW : " + URLsToSearch.size() );

        if ( URLsToSearch.size() != 0 ) {

            /* for debugging purposes */
            System.out.println("Crawling again : ");

            /* crawling the newly fetched URLs*/
            getUrls ( URLsToSearch, getQueryString() );

        }

    } // end function getUrls

    //--------------------------------------------------------------------------
    /** Javadoc here */
    private static String getAbsoluteURLName ( String baseURL, String relativeURL ) {

        String returnURL = null;

        if ( relativeURL.startsWith( RELATIVE_URL_START ) ) {

            returnURL = HTTP_START_TAG + baseURL.split("/")[2].toString().concat( relativeURL );

        } else {

            returnURL = baseURL.concat( relativeURL );
            
        }

        /* for debugging purposes */
        //System.out.println("return URL : " + returnURL );

        return returnURL;
        
    } // end function getAbsoluteURLName

    //--------------------------------------------------------------------------
    /** Javadoc here */
    private static boolean getURLContentType ( String urlString ) {

        boolean returnValue = false;

        try {

            Thread.sleep(SLEEP_MILLISECONDS);

            URL url = new URL ( urlString );

            URLConnection connection = url.openConnection();

            connection.setReadTimeout(TIMEOUT_MILLISECONDS);

            String contentType = connection.getContentType();

            /* for debugging purposes */
            System.out.println("url : " + urlString + " | ContentType : " + contentType);

            if ( AUDIO_CONTENT_TYPE.contains ( contentType ) ) {

                audioUrls.add( urlString );

            }
            else if ( VIDEO_CONTENT_TYPE.contains( contentType ) ) {

                videoUrls.add( urlString );

            }
            else if ( HTML_CONTENT_TYPE.contains ( contentType ) ) {

                returnValue = true;

            } else {

                    /**
                     *  This is just for debugging purposes
                     *  to make sure we handle all Content
                     *  types of our interest. This will be reported
                     */
                    //System.out.println("Content Type not handled - " + contentType);

            }

            return returnValue;
            
        } catch ( Exception e ) {

            Console.error( "BaseMediaHandler.getURLContentType", "Java Exception - " + e.getMessage() );

            return false;
        }

    } // end function getURLContentType

    //--------------------------------------------------------------------------
     /** Javadoc here */
    public static String getQueryString () {

        return searchString;

    } // end function getQuery

    //--------------------------------------------------------------------------
    /** Javadoc here */
    public static void setQueryString ( String string ) {

        searchString = string;

    } // end function setQueryString

    //--------------------------------------------------------------------------
    /** Javadoc here */
    private static boolean doesURLContainsSearchString ( String urlString ) {

        String decodeUrl = URLDecoder.decode(urlString);

        //System.out.println("url decoded to  : " + decodeUrl );

        String[] searchStringArray = searchString.split( SPLIT_WITH_SPACE );

        for ( int i = 0; i < searchStringArray.length; i ++ ) {

            if ( ! decodeUrl.toLowerCase().contains( searchStringArray[i].toLowerCase() ) ) {

                //System.out.println("url : " + urlString + " doesn't contains");
                return false;

            }
        }

        System.out.println("url : " + urlString );
        
        return true;
    }

    public void query(String url, String query) {
        throw new UnsupportedOperationException("Not supported yet.");
    }
} // end class BaseMediaHandler
